package com.yq.core.test;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.regex.Matcher;

import com.alibaba.fastjson.JSON;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.cookie.Cookie;
import org.apache.http.cookie.CookieOrigin;
import org.apache.http.cookie.CookieSpec;
import org.apache.http.cookie.CookieSpecProvider;
import org.apache.http.cookie.MalformedCookieException;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.cookie.BestMatchSpecFactory;
import org.apache.http.impl.cookie.BrowserCompatSpec;
import org.apache.http.impl.cookie.BrowserCompatSpecFactory;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class Crawler {

    //public static String pageNo = "1";

    public static void main(String[] args) {

        getHtml("http://weibo.com/p/1005052165485522/follow?relate=fans");

        /*try {
            Map<String, String> map = new HashMap();
            map.put("SSOLoginState","1499995770");
            map.put("un",           "151775096@qq.com");
            map.put("UOR",          ",weibo.com,site.baidu.com");
            map.put("TC-Page-G0",   "0dba63c42a7d74c1129019fa3e7e6e7c");
            map.put("SINAGLOBAL",   "1368203423917.2935.1493175400986");
            map.put("SCF",          "Aom3QkTXbwDXKKqSDl-PF3P2QXGEjV6gs8HLOq-5MBesCorR858imhvGlEY15R2CTcnv5i8FcVA7sNybiuF-PA0.");
            map.put("SUB",          "_2A250bG4qDeRhGedG6VYS8S3FzzuIHXVXGNjirDV8PUNbmtAKLXikkW-FDxnD50TqMn-WAvWa8f6o2NMSVw..");
            map.put("SUBP",         "0033WrSXqPxfM725Ws9jqgMF55529P9D9WWrD7giRw76kzaAQLVqT_MD5JpX5KMhUgL.Fo2ReoB0eKe4ShM2dJLoIp-LxK-LBo5LBo.LxKqL1-BL1he_gH2t");
            map.put("SUHB",         "05iLUzln45PsAi");
            map.put("ALF",          "1531531769");
            map.put("TC-Ugrow-G0",  "5e22903358df63c5e3fd2c757419b456");
            map.put("_s_tentry",    "-");
            map.put("Apache",       "7491665433626.622.1499995761347");
            map.put("ULV",          "1499995761384:6:3:3:7491665433626.622.1499995761347:1499931365460");
            map.put("TC-V5-G0",     "31f4e525ed52a18c5b2224b4d56c70a1");
            map.put("YF-Page-G0",   "091b90e49b7b3ab2860004fba404a078");
            map.put("YF-V5-G0",     "a1eb9ea7ccc47f9e865137ccb4cf9f3");

            Document doc = Jsoup.connect("http://weibo.com/p/1005052165485522/follow?relate=fans")
                    .cookies(map)
                    .get();

            Elements elements = doc.select("body").select("script");
            for(int i=0; i<elements.size(); i++) {
                Element element = elements.get(i);
                String scriptStr = element.toString();

                //if(scriptStr.indexOf("<script>FM.view({\"ns\":\"pl.third.information.index\"")!=0) {
                if(scriptStr.indexOf("<script>FM.view({\"ns\":\"pl.content.followTab.index\"")!=0) {
                    continue;
                }
                String jsonStr = scriptStr.replace("<script>FM.view(", "").replace(");</script>", "").replace(")</script>", "");

                Map<String, String> scriptMap = (Map<String, String>) JSON.parse(jsonStr);
                Document div = Jsoup.parse(scriptMap.get("html"));

                System.out.println(div.select("a.page.S_txt1"));

                getInfo(div.select("li.follow_item.S_line2"));

                //System.out.println(div);
                //System.out.println(div.select("li.follow_item.S_line2"));//
                //System.out.println(div.getElementsByTag("dt"));

            }


        } catch (IOException e) {
            e.printStackTrace();
        }*/

    }

    public static void getHtml(String url) {
        try {
            Map<String, String> map = new HashMap();
            map.put("SSOLoginState","1499995770");
            map.put("un",           "151775096@qq.com");
            map.put("UOR",          ",weibo.com,site.baidu.com");
            map.put("TC-Page-G0",   "0dba63c42a7d74c1129019fa3e7e6e7c");
            map.put("SINAGLOBAL",   "1368203423917.2935.1493175400986");
            map.put("SCF",          "Aom3QkTXbwDXKKqSDl-PF3P2QXGEjV6gs8HLOq-5MBesCorR858imhvGlEY15R2CTcnv5i8FcVA7sNybiuF-PA0.");
            map.put("SUB",          "_2A250bG4qDeRhGedG6VYS8S3FzzuIHXVXGNjirDV8PUNbmtAKLXikkW-FDxnD50TqMn-WAvWa8f6o2NMSVw..");
            map.put("SUBP",         "0033WrSXqPxfM725Ws9jqgMF55529P9D9WWrD7giRw76kzaAQLVqT_MD5JpX5KMhUgL.Fo2ReoB0eKe4ShM2dJLoIp-LxK-LBo5LBo.LxKqL1-BL1he_gH2t");
            map.put("SUHB",         "05iLUzln45PsAi");
            map.put("ALF",          "1531531769");
            map.put("TC-Ugrow-G0",  "5e22903358df63c5e3fd2c757419b456");
            map.put("_s_tentry",    "-");
            map.put("Apache",       "7491665433626.622.1499995761347");
            map.put("ULV",          "1499995761384:6:3:3:7491665433626.622.1499995761347:1499931365460");
            map.put("TC-V5-G0",     "31f4e525ed52a18c5b2224b4d56c70a1");
            map.put("YF-Page-G0",   "091b90e49b7b3ab2860004fba404a078");
            map.put("YF-V5-G0",     "a1eb9ea7ccc47f9e865137ccb4cf9f3");

            Document doc = Jsoup.connect(url)
                    .cookies(map)
                    .get();

            Elements elements = doc.select("body").select("script");
            for(int i=0; i<elements.size(); i++) {
                Element element = elements.get(i);
                String scriptStr = element.toString();

                //if(scriptStr.indexOf("<script>FM.view({\"ns\":\"pl.third.information.index\"")!=0) {
                if(scriptStr.indexOf("<script>FM.view({\"ns\":\"pl.content.followTab.index\"")!=0) {
                    continue;
                }
                String jsonStr = scriptStr.replace("<script>FM.view(", "").replace(");</script>", "").replace(")</script>", "");

                Map<String, String> scriptMap = (Map<String, String>) JSON.parse(jsonStr);
                Document div = Jsoup.parse(scriptMap.get("html"));

                int size = div.select("a.page.S_txt1").size();

                getInfo(div.select("li.follow_item.S_line2"));

                String pageNo = div.select("a.page.S_txt1.S_bg1").text();
                String pageTotal = div.select("a.page.S_txt1").get(size-2).text();

                System.out.println("当前页 = " + pageNo + "/" +pageTotal);
                System.out.println("--------------------");

                if(!pageNo.equals(pageTotal)) {
                    Thread.sleep(30000);
                    getHtml("http://weibo.com/p/1005052165485522/follow?relate=fans&page="+(Integer.parseInt(pageNo)+1));
                }

                //System.out.println(div);
                //System.out.println(div.select("li.follow_item.S_line2"));//
                //System.out.println(div.getElementsByTag("dt"));
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public static void getInfo(Elements elements) {
        for(int i=0; i<elements.size(); i++) {
            Element element = elements.get(i);

            System.out.println("uid, 昵称，性别 = " + element.attr("action-data"));
            System.out.println("头像 = " + element.select("dt.mod_pic img").attr("src"));
            System.out.println("地址 = " + element.select("div.info_add span").text());
            System.out.println("简介 = " + element.select("div.info_intro span").text());
            System.out.println("途径 = " + element.select("div.info_from").text());

            System.out.println("======================================================");
        }
    }

}
